#!/usr/bin/perl -w
#引擎:补天360  http://loudong.360.cn/vul/list-more/state/8/page/?
use DBI;
use warnings;
use Encode;
use utf8;
binmode(STDIN,':encoding(utf8)');
binmode(STDOUT,':encoding(utf8)');
binmode(STDERR,':encoding(utf8)');  
use open ":encoding(utf8)";
use LWP::Protocol::https;
use LWP::UserAgent;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use warnings;
use Time::Local;
use Date::Parse;

&insert_news(1);

sub insert_news(){

    my $page= $_[0];
	my $flag=0;
    my $count;
	my $driver="DBI:mysql";
	my $database="news";
	my $host="localhost";
	my $user="root";
	my $password="192510";
	my $db=DBI->connect("$driver:database=$database;host=$host;user=$user;password=$password");
	
	#得到前几天的时间
	my $current_time=$db->prepare("select UNIX_TIMESTAMP(date_sub(curdate(),interval 7 day))");
	$current_time->execute();
	my @current_time2=$current_time->fetchrow_array;
	my $current_time3=$current_time2[0];
	
	my $url="http://loudong.360.cn/vul/list-more/state/8/page/$page";
	my ($ua, $request , $response , $content);
	$ua = LWP::UserAgent->new;
	my $agentstr=&randomagent;
	$ua->agent('User-Agent'=>$agentstr);
	$ua->proxy(['http', 'ftp'], 'http://ymap.ydscan.net:8188');
	$request = HTTP::Request->new('GET',$url);
	$response = $ua->request($request);
	if($response->code ne '200'){
	
	}else{
	     $content = $response->content;
	     my $html=decode("utf-8",$content);
		 
		 my @list_viewarray = $html =~ m/<p class="list-view">.*<\/p>/g;
		my $array_num=scalar @list_viewarray;
		for(my $i=0; $i<$array_num;$i++){
			my $authortitlehref= $list_viewarray[$i];
			#提交者
			my @autorarray=split(/发布/,$authortitlehref);
			my $autorstr1=$autorarray[0];
			$autorstr1=~s/<p class="list-view">//g;
			my $rel_author="";
			if($autorstr1=~m/<\/a>/){
			   my @author4=split(/">/,$autorstr1);
			   my $author5=$author4[1];
			   $author5=~s/<\/a>//g;
			   $rel_author=$author5;
			}else{
				$rel_author=$autorstr1;
			}
			#链接和标题
			my $aherfstr=$autorarray[1];
			my @titlehrefarray = $aherfstr =~ m/<a href=.*<\/a>/g;
			my $hrefstr1=$titlehrefarray[0];
			my $rel_href="";
			my $rel_title="";
			if($hrefstr1=~m/">/){
				my @aherfarray=split(/">/,$hrefstr1);
				$rel_href=$aherfarray[0];
				if($rel_href=~m/<a href="/){
					$rel_href=~s/<a href="//g;
				}
				$rel_href="http://loudong.360.cn".$rel_href;
				#标题
				$rel_title=$aherfarray[1];
				if($rel_title=~m/<\/a>/){
					$rel_title=~s/<\/a>//g;
				}
			}
			
			
			#提交日期
			my @publishtimearray=split(/$authortitlehref/,$html);
			my $rel_publishtime="";
			if($#publishtimearray>0){
				my @publisharray1=split(/<p>/,$publishtimearray[1]);
				if($#publisharray1>0){
					my @publisharray2=split(/<span class="color_td1">/,$publisharray1[1]);
					if($#publisharray2>0){
						$rel_publishtime=$publisharray2[0];
						$rel_publishtime=~s/年/-/g;
						$rel_publishtime=~s/月/-/g;
						$rel_publishtime=~s/日//g;
					}
					
				}
			}
			
			#其它
			my $web=4;
			my $rel_content="无";
			my $m=$rel_title.$rel_author;
			Encode::_utf8_off($m);
			my $md5=md5($m);
		    my $bj_publishtime=str2time("$rel_publishtime GMT"); 
			
			
			# my ($ua, $request , $response , $content);
			# $ua = LWP::UserAgent->new;
			# $request = HTTP::Request->new('GET',$rel_href);
			# $response = $ua->request($request);
			# $ua->proxy(['http', 'ftp'], 'http://ymap.ydscan.net:8188');
			# $content2 = $response->content;
			# my $html2=decode("utf-8",$content2);
			# my @title=split(/<font style="font-weight: bold;">/,$html2);
			# my @title2=split(/<\/font>/,$title[1]);
			# my $rel_title=$title2[0];  
				
			if($bj_publishtime>=$current_time3){#说明是最近7天之内的新闻
					  
				  my $q_rel_title=$rel_title;
				  $q_rel_title=~s/\s//g;
				  
				   
				  my $q_rel_href=$rel_href;
				  $q_rel_href=~s/\s//g;
				  
				  my $q_rel_author=$rel_author;
				  $q_rel_author=~s/\s//g;
				
				  my $cf_count=0;
				  
				 if($q_rel_title ne"" and $q_rel_href ne"" and $q_rel_author ne""){
						
						my $sth = $db->prepare(q{insert ignore into news(title,web,author,publishtime,content,link,nmd5)VALUES(?,?,?,?,?,?,?)}) or die $db->errstr;
						$sth->execute($rel_title,$web,$rel_author,$rel_publishtime,$rel_content,$rel_href,$md5) or die $db->errstr;
						print "title:$q_rel_title, href:$q_rel_href, autor:$q_rel_author\n";
				  }else{
						print "有空值 title:$q_rel_title, href:$q_rel_href, autor:$q_rel_author\n";
				  }
						  
						  
			}else{#不是最近7天的
				
				 return;
				
		    }
		
	
	  } 
	     sleep(5);
	     $page=$page+1;
	     &insert_news($page);
	
	}
	
}
#模拟随机浏览器
sub randomagent
{
## FireFox
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1
# Chrome
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1
##Safari
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13
##Sogou
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
##Baidu
# Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7
##QQ
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.7 (KHTML, like Gecko) Chrome/20.0.1099.0 Safari/536.7 QQBrowser/6.14.15138.201
##Maxthon
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12
##LBBROWSER
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER
##TheWorld
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD
	my @agent_conf=('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
	'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
	'Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD');
	my $int=int(rand(12));
	return $agent_conf[$int];
}

